package com.duowan.mspider.processor;

import com.duowan.mspider.util.MyHttpClient;
import com.duowan.mspider.util.Constant;
import com.duowan.mspider.util.Constant.KeyWordsFile;
import com.duowan.mspider.util.Constant.SearchEngine;
import com.duowan.mspider.util.FileUtils;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

import java.io.File;
import java.net.URLDecoder;
import java.util.List;

/**
 * @author:wangyan1
 * @create_time: 2018-04-10 22:09
 */

public class SearchEngineProcessor implements PageProcessor {

	private Constant.SearchEngine engine;
	private KeyWordsFile keyWordsFile;

	@Override
	public void process(Page page) {
		List<String> listUrls = page.getHtml().xpath(engine.xpath).all();
		if (CollectionUtils.isEmpty(listUrls)) return;

		for (String url : listUrls) {
			if (StringUtils.isBlank(url)) continue;
			try {
				if (engine == SearchEngine.Sogou) {
					url = MyHttpClient.getSougouRealUrlInfo(URLDecoder.decode("http:" + url, "UTF-8"));
				}
				String link = URLDecoder.decode(url, "UTF-8");
				if (StringUtils.isBlank(link) || link.startsWith("https")) continue;

				FileUtils.write(new File(keyWordsFile.getTempFile()), link);
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
	}

	@Override
	public Site getSite() {
		return Site.me().setRetryTimes(3).setSleepTime(1000)
					   .setUserAgent("Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) " +
											 "AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166  Safari/535.19");
	}

	public SearchEngineProcessor(SearchEngine engine, KeyWordsFile keyWordsFile) {
		this.engine = engine;
		this.keyWordsFile = keyWordsFile;
	}

	public SearchEngine getEngine() {
		return engine;
	}


	public static void main(String[] args) throws Exception {

		for (KeyWordsFile keyWordsFile : Constant.KeyWordsFile.values()) {
			//先删除temp文件
			File tempFile = new File(keyWordsFile.getTempFile());
			if (tempFile.exists()) tempFile.delete();

			List<String> keywords = FileUtils.read(new File(keyWordsFile.srcFileName));
			for (SearchEngine searchEngine : SearchEngine.values()) {
				if (!searchEngine.valid) continue;

				new Thread(new Runnable() {
					@Override
					public void run() {
						SearchEngineProcessor processor = new SearchEngineProcessor(searchEngine, keyWordsFile);
						Spider spider = Spider.create(processor);
						for (String keyword : keywords) {
							spider.addUrl(searchEngine.url + keyword);
						}
						spider.thread(5).run();
					}
				}).start();
			}
		}
	}

}
